Stable Diffusion#

Note

Install ekorpkit package first.

Set logging level to Warning, if you don’t want to see verbose logging.

If you run this notebook in Colab, set Hardware accelerator to GPU.

!pip install ekorpkit[art] exit()

from ekorpkit import eKonf

eKonf.setLogger("INFO")
eKonf.set_cuda(device="4,5")
print("version:", eKonf.__version__)

is_notebook = eKonf.is_notebook()
is_colab = eKonf.is_colab()
print("is notebook?", is_notebook)
print("is colab?", is_colab)
if is_colab:
    eKonf.mount_google_drive(
        workspace="MyDrive/colab_workspace", project="disco-imagen"
    )

print("environment variables:")
eKonf.print(eKonf.env().dict())
INFO:ekorpkit.base:Setting cuda device to ['A100-SXM4-40GB (id:4)', 'A100-SXM4-40GB (id:5)']
INFO:ekorpkit.utils.notebook:shell type: ZMQInteractiveShell
INFO:ekorpkit.utils.notebook:Google Colab not detected.
version: 0.1.40.post0.dev1
is notebook? True
is colab? False
environment variables:
{'CUDA_DEVICE_ORDER': 'PCI_BUS_ID',
 'CUDA_VISIBLE_DEVICES': '4, 5',
 'EKORPKIT_CONFIG_DIR': '/workspace/projects/ekorpkit-book/config',
 'EKORPKIT_DATA_DIR': None,
 'EKORPKIT_LOG_LEVEL': 'INFO',
 'EKORPKIT_PROJECT': 'ekorpkit-book',
 'EKORPKIT_WORKSPACE_ROOT': '/workspace',
 'KMP_DUPLICATE_LIB_OK': 'TRUE',
 'NUM_WORKERS': 230}

Create a stable diffusion instance#

cfg = eKonf.compose("model/stable_diffusion")
sd = eKonf.instantiate(cfg)
INFO:ekorpkit.base:Loaded .env from /workspace/projects/ekorpkit-book/config/.env
INFO:ekorpkit.utils.notebook:shell type: ZMQInteractiveShell
INFO:ekorpkit.base:setting environment variable CACHED_PATH_CACHE_ROOT to /workspace/.cache/cached_path
INFO:ekorpkit.base:setting environment variable KMP_DUPLICATE_LIB_OK to TRUE
INFO:absl:Unable to initialize backend 'tpu_driver': NOT_FOUND: Unable to find driver in registry given worker: 
INFO:absl:Unable to initialize backend 'rocm': NOT_FOUND: Could not find registered platform with name: "rocm". Available platform names are: Interpreter Host CUDA
INFO:absl:Unable to initialize backend 'tpu': module 'jaxlib.xla_extension' has no attribute 'get_tpu_client'
INFO:ekorpkit.models.art.base:> downloading models...
INFO:ekorpkit.models.art.base:> loading modules...
INFO:ekorpkit.models.art.base:No modules to load
INFO:ekorpkit.models.art.base:> loading models...

Generate images#

text_prompts = "a living tiger inside a gorgeous cave in a futuristic Jeju island"
batch_name = "cave-jeju"
sd.imagine(
    text_prompts, 
    batch_name=batch_name, 
    num_samples=2, 
    show_collage=True,
    guidance_scale=2
)
INFO:ekorpkit.models.art.base:Merging config with args: {}
INFO:ekorpkit.models.art.base:Prompt: None
../../../../_images/stable-diffusion_8_1.png
INFO:ekorpkit.models.art.base:Saving config to /workspace/projects/ekorpkit-book/stable-diffusions/outputs/stable-diffusion/cave-jeju/cave-jeju(1)_settings.yaml
{'image_filepaths': ['/workspace/projects/ekorpkit-book/stable-diffusions/outputs/stable-diffusion/cave-jeju/cave-jeju(1)_0000.png',
  '/workspace/projects/ekorpkit-book/stable-diffusions/outputs/stable-diffusion/cave-jeju/cave-jeju(1)_0001.png'],
 'config_file': 'cave-jeju(1)_settings.yaml',
 'config': {'batch_name': 'cave-jeju',
  'mode': 'generate',
  'text_prompts': {0: ['a living tiger inside a gorgeous cave in a futuristic Jeju island']},
  'batch_num': 1,
  'num_samples': 2,
  'num_images_per_prompt': 1,
  'width': 768,
  'height': 512,
  'num_inference_steps': 100,
  'guidance_scale': 2,
  'init_image': None,
  'mask_image': None,
  'inpaint_strength': 0.75,
  'set_seed': 'random_seed',
  'seed': 710849687,
  'resume_run': False,
  'run_to_resume': 'latest'}}

Inpaint images#

base_url = "https://github.com/entelecheia/ekorpkit-book/raw/main/assets/figs"
init_image_path = f"{base_url}/chu-horse.png"
mask_image_path = f"{base_url}/chu-horse-masking-rough.png"
mask_image_exact_path = f"{base_url}/chu-horse-masking-exact.png"

init_image = eKonf.load_image(init_image_path)
mask_image = eKonf.load_image(mask_image_path)
mask_image_exact = eKonf.load_image(mask_image_exact_path)

sd.compare_images([mask_image, init_image, mask_image_exact], resize_ratio=0.5)
../../../../_images/stable-diffusion_10_0.png
prompt = "a golden statue of a eagle"

images = sd.imagine(
    prompt, 
    mode="inpaint",
    init_image=init_image_path,
    mask_image=mask_image_path,
    num_samples=2,
    num_images_per_prompt=2, 
    guidance_scale=3, 
    num_inference_steps=50,
    return_including_init_image=False
)
INFO:ekorpkit.models.art.base:Merging config with args: {}
INFO:ekorpkit.models.art.base:Prompt: None
../../../../_images/stable-diffusion_11_1.png

Stitch images#

sd.imagine(
    text_prompts, 
    mode="stitch",
    batch_name=batch_name, 
    num_samples=6, 
    show_collage=True,
)
INFO:ekorpkit.utils.notebook:shell type: ZMQInteractiveShell
../../../../_images/stable-diffusion_13_1.png
INFO:ekorpkit.models.art.stable: >> elapsed time to imagine: 0:01:39.323883
INFO:ekorpkit.models.art.base:Saving config to /workspace/projects/ekorpkit-book/stable-diffusions/outputs/stable-diffusion/cave-jeju/cave-jeju(4)_settings.yaml
{'stitched_image_path': '/workspace/projects/ekorpkit-book/stable-diffusions/outputs/stable-diffusion/cave-jeju/cave-jeju(4)_panorama.png',
 'image_filepaths': ['/workspace/projects/ekorpkit-book/stable-diffusions/outputs/stable-diffusion/cave-jeju/cave-jeju(4)_0000.png',
  '/workspace/projects/ekorpkit-book/stable-diffusions/outputs/stable-diffusion/cave-jeju/cave-jeju(4)_0001.png',
  '/workspace/projects/ekorpkit-book/stable-diffusions/outputs/stable-diffusion/cave-jeju/cave-jeju(4)_0002.png',
  '/workspace/projects/ekorpkit-book/stable-diffusions/outputs/stable-diffusion/cave-jeju/cave-jeju(4)_0003.png',
  '/workspace/projects/ekorpkit-book/stable-diffusions/outputs/stable-diffusion/cave-jeju/cave-jeju(4)_0004.png',
  '/workspace/projects/ekorpkit-book/stable-diffusions/outputs/stable-diffusion/cave-jeju/cave-jeju(4)_0005.png'],
 'config_file': 'cave-jeju(4)_settings.yaml',
 'config': {'batch_name': 'cave-jeju',
  'mode': 'Stitch',
  'text_prompts': {0: ['a living tiger inside a gorgeous cave in a futuristic Jeju island']},
  'batch_num': 4,
  'num_samples': 6,
  'num_images_per_prompt': 1,
  'width': 768,
  'height': 512,
  'num_inference_steps': 100,
  'guidance_scale': None,
  'init_image': None,
  'mask_image': None,
  'inpaint_strength': 0.75,
  'set_seed': 'random_seed',
  'seed': 501160171,
  'resume_run': False,
  'run_to_resume': 'latest'}}

To view the panorama, upload the image to renderstuff.

Show config#

sd.show_config(batch_name=batch_name, batch_num=0)
INFO:ekorpkit.models.art.base:Loading config from /workspace/projects/ekorpkit-book/stable-diffusions/outputs/stable-diffusion/cave-jeju/cave-jeju(0)_settings.yaml
INFO:ekorpkit.models.art.base:Merging config with diffuse defaults
INFO:ekorpkit.models.art.base:Merging config with args: {}
{'batch_name': 'cave-jeju',
 'batch_num': 3,
 'guidance_scale': None,
 'height': 512,
 'init_image': None,
 'inpaint_strength': 0.75,
 'mask_image': None,
 'mode': 'Stitch',
 'num_images_per_prompt': 1,
 'num_inference_steps': 100,
 'num_samples': 6,
 'resume_run': False,
 'run_to_resume': 'latest',
 'seed': 2034638016,
 'set_seed': 'random_seed',
 'text_prompts': {0: ['a living tiger inside a gorgeous cave in a futuristic '
                      'Jeju island']},
 'width': 768}